<?php
	/**
	 * 由js调用抓取网页并分析后入库
	 */
	require_once '../lib/controller.php';

	class action extends controller
	{
		var $db = '';
		var $site_id = '';
		var $content_type = 'text/html';
		var $charset = 'utf-8';
		var $catch_remote_cache_expire_time = 3600;

		function action()
		{
			parent::__construct();
			$this->db = $this->module('lib.db');
		}

		function do_get()
		{
			//检查站点是否有记录
			$check_data = $this->check_list();

			//判断是否要刷新记录列表：若非第一次抓取
			if( is_array( $check_data ) )
			{
				$this->site_id = $check_data['site_id'];
				$list_id = $check_data['id'];
				if( ( time() - $check_data['ctime'] ) < $this->catch_remote_cache_expire_time && ! isset( $this->get['nocache'] ) )
				{
					$s = 'Tips: Content still be cached';
				}
				else	//缓存已经超时
				{
					$data = $this->get_new_list();
					//若内容有修改过
					if( md5(implode('', $data['txt']['url'] )) != $check_data['content_hash'] )
					{
						if(  ! empty( $this->site_id ) && is_numeric($this->site_id) )
						{
							$list_id = $this->add_list($data, $check_data['id']);
						}
						$s = 'Tips: Refresh cache';
					}
					else
					{
						$s = 'Tips: Cache expired, but content does not change';
					}
				}
			}
			else	//若第一次抓取
			{
				$data = $this->get_new_list();
				if(  ! empty( $this->site_id ) && is_numeric($this->site_id) )
				{
					$list_id = $this->add_list($data);
				}
				$s = 'Tips: New content and generate cache';
			}

			//若站点有收录
			if(  ! empty( $this->site_id ) && is_numeric( $this->site_id ) )
			{
				$data1 = $this->get_list( $list_id );
			}
			else
			{
				$data1 = $data;
			}

			if( ! $data1 )
			{
				die( $this->_lang['no_data'] );
			}
			else
			{
				if(  ! empty( $this->site_id ) && is_numeric($this->site_id) )
				{
					$show = $this->show_format($data1);
				}
				else
				{
					$show = $this->anonymouse_show_format($data1);
				}

				echo $show;
				echo "\r\n";
				echo json_encode( array('status'=>$s . ' on ' . date('Y-m-d H:i:s'), 'is_preview'=>empty($this->site_id) ? 1 : 0, 'uri'=>urlencode($this->get['uri'])) );
			}

			$this->site_visit_log();
		}

		public function check_get()
		{
			$tool = $this->module( 'lib.tool' );
			if( ! $tool->check_url($this->get['uri']) )
			{
				echo $this->_lang['wrong_url_format'];
				return false;
			}

			$sql = "select id from meye_site where url_hash = '" . md5($this->get['uri']) . "' ";
			$data = $this->db->query( $sql, 'array', true );
			if( isset( $data['id'] ) )
			{
				$this->site_id = $data['id'];
			}

			return true;
		}

		function get_new_list()
		{
			include_once '../biz/snap.php';

			$action = new snap($this->get['uri']);
			$data = $action->run();

			$this->content_type = $action->content_type;
			$this->charset = $action->charset;

			return $data;
		}


		function show_format($data)
		{
			$new_cate_list = $this->get_cate($this->site_id);
			$id_cnamd_map = $this->tool_map($new_cate_list, 'id', 'id');
			if( isset($this->session['user_id']) )
			{
				$hidden_cate_list = $this->get_hidden_cate($this->site_id, $this->session['user_id']);
				if( strpos($hidden_cate_list, ',') !== false )
				{
					$hidden_cate_list_arr = explode(',', $hidden_cate_list);
				}
				else
				{
					$hidden_cate_list_arr = array($hidden_cate_list);
				}
			}
			else
			{
				$hidden_cate_list_arr = array('');
			}

			$s = '';
			$backup_cate_id = '';
			foreach( $data as $key=>$val )
			{
				if( empty($val['title']) )	//认为是图片
				{
					if ( empty( $val['pic_src'] ) )
					{
						continue;	//为何有空内容?
					}
					$val['url'] = html_entity_decode( $val['url'], ENT_QUOTES);
					$val['pic_src'] = html_entity_decode($val['pic_src'], ENT_QUOTES);
					$s .= '<a href="article.php?id=' . $val['id'] . '&uri=' . urlencode($val['url'])  . '" target="_blank"><img src="img.php?uri=' . urlencode($val['pic_src']) . '" border="0"></a>' . "\r\n";
				}
				else
				{
					if( in_array($val['cate_id'], $hidden_cate_list_arr) )
					{
						continue;
					}
					if( $backup_cate_id != $val['cate_id'] )
					{
						if( $val['cate_id'] == 0 )
						{
							$s .= "Unkown cate_id\r\n";
						}
						else
						{
							if( isset( $id_cnamd_map[$val['cate_id']] ) )
							{
								$s .= $id_cnamd_map[$val['cate_id']] . "|" . $this->site_id."\r\n";
							}
						}
					}
					$val['title'] = html_entity_decode($val['title'], ENT_QUOTES);
					$val['url'] = html_entity_decode($val['url'], ENT_QUOTES);
					$s .= '<a href="article.php?id=' . $val['id'] . '&uri=' . urlencode($val['url'])  . '" target="_blank">' . $val['title'] . '</a>' . "\r\n";
					$backup_cate_id = $val['cate_id'];
				}
			}
			$s = substr($s, 0, -2);

			return $s;
		}

	/**
	 * 未收录站点显示格式
	 * detail description
	 * @param array $data
	 * @return string
	 */
		function anonymouse_show_format($data)
		{
			$s = '';

			foreach($data['img']['url'] as $key=>$url)
			{
				if ( empty( $data['img']['src'][$key]  ) )
				{
					continue;
				}

				$s .= '<a href="' . $url  . '" target="_blank"><img src="img.php?uri=' . urlencode($data['img']['src'][$key]) . '" border="0"></a>' . "\r\n";
			}
			foreach($data['txt']['title'] as $key=>$title)
			{
				$title = mb_convert_encoding($title, 'utf-8', $data['charset']);
				$url = urldecode( $data['txt']['url'][$key] );
				$s .= '<a href="' . $url . '" target="_blank">' . $title . '</a>' . "\r\n";
			}
			$s = substr($s, 0, -2);

			return $s;
		}

		function get_cate( $site_id )
		{
			$sql = "select id,site_id,cname,url,url_hash from meye_list_cate where site_id='$site_id' ";
			$data = $this->db->query( $sql, 'all', true );

			return $data;
		}

		function get_hidden_cate($site_id, $user_id)
		{
			$sql = "select hide_cate_id_list from meye_setting where site_id='$site_id' and user_id='$user_id' ";
			$data = $this->db->query( $sql, 'all', true );
			if( ! isset($data['hide_cate_id_list']) )
			{
				$ret = false;
			}
			else
			{
				$ret = $data['hide_cate_id_list'];
			}

			return $ret;
		}

		function parse_cate($data)
		{
			$ret_url = array();
			foreach( $data as $key=>$url )
			{
				$parsed_path = '';
				$url_data = parse_url($url);
				if( isset($url_data['path']) )
				{
					if( strpos($url_data['path'], '/') !== false )
					{
						$url_data['path'] = preg_replace( "/&.*/", "", $url_data['path'] );	//先去掉影响分析的参数
						$url_data['path'] = substr($url_data['path'], 0, strrpos($url_data['path'], '/') );
						$slash_token = explode('/', $url_data['path']);
						foreach( $slash_token as $token )
						{
							if( empty($token) || $token == '/' ) continue;
							if( preg_match("/[0-9\-_]{1,}/", $token) )
							{
								continue;
							}
							else
							{
								$parsed_path .= $token . '/';
							}
						}
					}
					else
					{
						$parsed_path = $url_data['path'];
					}

				}
				$regular_url = $url_data['scheme'] . '://' . $url_data['host'] . '/' . $parsed_path;
				if( ! in_array($regular_url, $ret_url) )
				{
					$ret_url[] = $regular_url;
				}
			}

			return $ret_url;
		}

		function update_cate( $url, $old_cate_list=array() )
		{
			$t = array();
			if( ! empty($old_cate_list) )
			{
				foreach($old_cate_list as $key=>$val)
				{
					$t[] = $val['url'];
				}
			}
			else
			{
				$t = array('');
			}
			$old_cate_list = $t;

			$txt_cate_list = $this->parse_cate($url['txt']['url']);
			$img_cate_list = $this->parse_cate($url['img']['url']);
			$cate_list = array_merge($txt_cate_list, $img_cate_list);
			$cate_list = array_unique($cate_list);
			$cnt = 1;
			foreach( $cate_list as $url )
			{
				if( ! in_array($url, $old_cate_list) )
				{
					$cname = $cnt;
					$url_hash = md5($url);
					$sql = "insert into meye_list_cate (site_id, cname, url, url_hash) values ('" . $this->site_id . "', '$cname', '$url', '$url_hash')";
					$this->db->query($sql);
					$cnt++;
				}
			}

			return true;
		}

		/**
		 * 检查是否有抓取记录
		 * 用来缓存判断
		 */
		function check_list()
		{
			$url_hash = md5($this->get['uri']);
			$sql = "select id,site_id,ctime,content_hash from meye_list where url_hash='" . $url_hash . "' ";
			$data = $this->db->query($sql, 'all');

			if( is_array($data) && isset($data[0]['id']) )
			{
				return $data[0];
			}
			else
			{
				return false;
			}
		}

		/**
		 * description
		 * detail description
		 * @param string user
		 * @param integer id
		 * @return int|string|array|object|mixed
		 */
		function get_list( $list_id )
		{
			$sql = "select id,title,pic_src,url,ctime,hot_flag,cate_id from meye_article where list_id='" . $list_id . "' ";
			$data = $this->db->query($sql, 'all');

			if( isset($data[0]['id']) )
			{
				return $data;
			}
			else
			{
				return false;
			}
		}

		function add_list($data, $list_id='')
		{
			if( ! is_array($data) || empty( $this->site_id ))
			{
				return false;
			}

			$this->db->begin_transaction();

			//开始 更新分类
			$old_cate_list = $this->get_cate($this->site_id);
			$cate_arr = $this->update_cate($data, $old_cate_list);

			$new_cate_list = $this->get_cate($this->site_id);
			$id_url_map = $this->tool_map($new_cate_list, 'url', 'id');
			//开始 更新列表及文章
			$ctime = time();
			$hot_flag = 0;
			$url = $this->get['uri'];
			$url_hash = md5($url);
			$site_id = $this->site_id;
			$content_hash = md5(implode('', $data['txt']['url'] ));

			$sql = "insert into meye_list (site_id, url, url_hash, ctime, content_hash) values ($site_id, '$url', '$url_hash', $ctime, '$content_hash')";
			$this->db->query($sql);
			$list_id = $this->db->get_insert_id();

			foreach($data['txt']['title'] as $key=>$title)
			{
				if( $data['charset'] != 'utf-8' )
				{
					$title = mb_convert_encoding( $title, 'UTF-8', $data['charset'] );
					//$title = iconv( $data['charset'], 'utf-8', $title );
				}
				//编辑器要用verdana字体或宋体，否则不能正常显示下面的点号。
				$title = preg_replace( '/‧/', '.', $title );
				$title = preg_replace( '/•/', '.', $title );
				$title = htmlspecialchars( $title, ENT_QUOTES );
				$url = mb_convert_encoding($data['txt']['url'][$key], 'utf-8', $data['charset']);
				$url = stripslashes( $url );
				$url = htmlspecialchars( substr($url, 0, 255), ENT_QUOTES );
				$regular_url = $this->parse_cate(array($url));
				$cate_id = $id_url_map[$regular_url[0]];
				if( empty( $cate_id ) )
				{
					//die('Unknown cate_id: ' . $regular_url[0] . print_r($id_url_map,true));
					continue;
				}
				$url_hash = md5($url);

				$sql = "insert into meye_article (site_id, list_id, title, pic_src, url, url_hash, ctime, hot_flag, cate_id) values ($site_id, $list_id, '$title', '', '$url', '$url_hash', $ctime, $hot_flag, $cate_id)";

				$this->db->query($sql);
			}

			foreach($data['img']['url'] as $key=>$url)
			{
				$regular_url = $this->parse_cate(array($url));
				$cate_id = $id_url_map[$regular_url[0]];
				if( empty($cate_id) )
				{
					continue;
					//die('Unknown cate_id: ' . $regular_url[0]);
				}

				$url = htmlspecialchars( urlencode( $url ), ENT_QUOTES );
				$url_hash = md5($url);
				$pic_src = $data['img']['src'][$key];
				$pic_src = htmlspecialchars( $pic_src, ENT_QUOTES );

				$sql = "insert into meye_article (site_id, list_id, title, pic_src, url, url_hash, ctime, hot_flag, cate_id) values ($site_id, $list_id, '', '$pic_src', '$url', '$url_hash', $ctime, $hot_flag, $cate_id)";
				$this->db->query($sql);
			}
			$this->db->commit_transaction();

			return $list_id;
		}

		function tool_map($data, $fkey, $fvalue)
		{
			$ret = array();
			if( ! empty($data) )
			{
				foreach($data as $key=>$val)
				{
					$ret[$val[$fkey]] = $val[$fvalue];
				}
			}

			return $ret;
		}

		function site_visit_log()
		{
			$url = substr($this->get['uri'], 0, 127);
			$ctime = time();
			$ip = ip2long($this->server['REMOTE_ADDR']);
			$ua = substr($this->server['HTTP_USER_AGENT'], 0, 127);
			$sql = "insert into meye_site_visit_log (url,ctime,ip,ua) values ('$url', $ctime, $ip, '$ua')";
			$this->db->query($sql);

			return true;
		}

	}

	$action = new action();
	$action->run();
?>